import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Vector;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.queryparser.flexible.standard.config.NumericFieldConfigListener;

public class DocsDatabase {

	public static final boolean DEV_MODE = true;
	private IndexWriter mIndexWriter;
	private Directory mIndex;
	Analyzer mAnalyzer = null;

	int mTotalNumOfDocument = 0;
	private TFIDFSimilarity mTfidfSimilarity = new MyTFIDFSimilarity();
	
	HashMap<Integer, Integer> m_termid_to_frquenecy = new HashMap<Integer, Integer>();
	Map<String, Integer> m_term_to_id = new HashMap<String, Integer>();
	Map<Integer, String> m_term_id_to_term = new HashMap<Integer, String>();
	

	public DocsDatabase(Vector<String> StopWordList,
			Boolean is_using_improved_algorithm) throws IOException {
		if (is_using_improved_algorithm) {
			mAnalyzer = new ImprovedAnalyzer(Version.LUCENE_48,
					StopFilter.makeStopSet(Version.LUCENE_48, StopWordList));
		} else {
			mAnalyzer = new StandardAnalyzer(Version.LUCENE_48,
					StopFilter.makeStopSet(Version.LUCENE_48, StopWordList));
		}

		IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_48,
				mAnalyzer);
		mIndex = new RAMDirectory();
		mIndexWriter = new IndexWriter(mIndex, config);
	}

	public void Init(String documents_file_full_path) throws IOException {
		readDocsFile(documents_file_full_path);
	}

	public void readDocsFile(String fileName) throws IOException {
		String line;
		File docsFile = new File(fileName);
		FileInputStream fileInputStream = new FileInputStream(docsFile);
		BufferedReader bufferedReader = new BufferedReader(
				new InputStreamReader(fileInputStream));

		// read the file line by line
		int i =0;
		while ((line = bufferedReader.readLine()) != null) {
			String[] parts = line.split("   ", 3);
			String id = parts[0].trim();
			int cluster_id =Integer.parseInt(parts[1].trim());
			String body = parts[2].trim();			
			addDoc(body, id, cluster_id);
			i++;
		}
		mTotalNumOfDocument = mIndexWriter.numDocs();
		
		mIndexWriter.close();
		bufferedReader.close();
		IndexReader reader = DirectoryReader.open(mIndex);
		Terms terms = SlowCompositeReaderWrapper.wrap(reader).terms("body"); 

	}

	public LuceneSearchResult search(String query) throws IOException,
			ParseException {
		Query q = new QueryParser(Version.LUCENE_48, "body", mAnalyzer)
				.parse(query);
		
		IndexReader reader = DirectoryReader.open(mIndex);
		IndexSearcher searcher = new IndexSearcher(reader);

		int hitsPerPage = reader.numDocs();
		searcher.setSimilarity(mTfidfSimilarity);
		TopScoreDocCollector collector = TopScoreDocCollector.create(
				hitsPerPage, true);
		searcher.search(q, collector);
		LuceneSearchResult result = new LuceneSearchResult(
				collector.topDocs().scoreDocs, searcher);
		return result;
	}

	private void addDoc(String body, String id, int gt_class_id) throws IOException {
		
		Document doc = new Document();

		FieldType type = new FieldType();
		type.setIndexed(true);
		type.setStored(true);
		type.setStoreTermVectors(true);
		
		//additional filters. Much easier than figuring out how to do it 
		//in lucene
		body = body.replaceAll("\\W", " ");
		body = body.replaceAll("_", " ");		
		body = body.replaceAll("[0-9]", "");
		
		Field body_field = new Field("body", body, type);
		
		doc.add(body_field);
		StringField field = new StringField("id", id, Field.Store.YES);		
		doc.add(field);
		IntField gt_class_it = new IntField("gt_class_id", gt_class_id, Field.Store.YES);
		doc.add(gt_class_it);
		mIndexWriter.addDocument(doc);
	}

	
	public int[] getDocIdToGTClusterId() throws IOException
	{
		 
		IndexReader reader = DirectoryReader.open(mIndex);
		int[] clustr_list = new int[reader.maxDoc()];
		for (int docID = 0; docID < reader.maxDoc(); docID++) {
			Document doc = reader.document(docID);
			int content = (Integer) doc.getField("gt_class_id").numericValue();
			clustr_list[docID] = content; 
		}
		return clustr_list;
	}
	
	public int[] getDocsId() throws IOException
	{
		 
		IndexReader reader = DirectoryReader.open(mIndex);
		int[] doc_id = new int[reader.maxDoc()];
		for (int docID = 0; docID < reader.maxDoc(); docID++) {
			Document doc = reader.document(docID);
			int content = Integer.parseInt(doc.getField("id").stringValue());
			doc_id[docID] = content; 
		}
		return doc_id;
	}
	

	public ArrayList<HashMap<Integer, Float>> createAndGetTfIdfMatrix() throws Exception
	{	
		ArrayList<HashMap<Integer, Float>> return_value = new ArrayList<HashMap<Integer,Float>>();
		IndexReader reader = DirectoryReader.open(mIndex);
		Map<String, Float> docFrequencies = new HashMap<String, Float>();
		MyTFIDFSimilarity tfidfSIM = new MyTFIDFSimilarity();
		
		TermsEnum termsEnumIt = MultiFields.getTerms(reader, "body").iterator(null);
		 
		//count the number of terms
		int number_of_terms =0 ;		
		while (termsEnumIt.next() != null) {
			++number_of_terms;
		}
		
		long number_of_docs = reader.maxDoc();
		if((number_of_terms != (int)number_of_terms) || (number_of_docs  !=(int)number_of_docs)) 
		{
			throw new Exception("error while trying to create  tf-idf matrix");
		}
		
		TermsEnum termEnum = MultiFields.getTerms(reader, "body")
				.iterator(null);
		BytesRef bytesRef;
		int term_id_i = 0;
		
		//create the term to term_id dictionary and compute each term frequency
		while ((bytesRef = termEnum.next()) != null) {
			if (termEnum.seekExact(bytesRef)) {
				String term = bytesRef.utf8ToString();
				
				float idf = tfidfSIM.idf(termEnum.docFreq(),
						reader.numDocs());
				docFrequencies.put(term, idf);
				m_term_to_id.put(term, term_id_i);
				m_term_id_to_term.put(term_id_i, term);
				m_termid_to_frquenecy.put(term_id_i, 0);
				term_id_i++;				
			}
		}

		for (int docID = 0; docID < reader.maxDoc(); docID++) 
		{
			HashMap<Integer, Float> tf_idf_vector = new HashMap<Integer, Float>();
			TermsEnum termsEnum = MultiFields.getTerms(reader, "body").iterator(null);
			DocsEnum docsEnum = null;
			
			Terms vector = reader.getTermVector(docID, "body");
			
			if(vector == null)
			{
				continue;
			}
			
			try {
				termsEnum = vector.iterator(termsEnum);
			} catch (NullPointerException e) {
				e.printStackTrace();
			}

			while ((bytesRef = termsEnum.next()) != null) {
				if (termsEnum.seekExact(bytesRef)) {
					String term = bytesRef.utf8ToString();
					float tf = 0;

					docsEnum = termsEnum.docs(null, null, DocsEnum.FLAG_FREQS);
					while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
						tf = tfidfSIM.tf(docsEnum.freq());
					}

					Float idf = docFrequencies.get(term);
					if(idf == null)
					{
						continue;
					}
					
					float w = tf * idf;
					Integer term_id = m_term_to_id.get(term);
					
					Integer term_freq =  m_termid_to_frquenecy.get(term_id);
					
					term_freq++;
					m_termid_to_frquenecy.put(term_id, term_freq);
					tf_idf_vector.put(term_id, w);
				}
			}
			return_value.add(tf_idf_vector);
		}
		
		return return_value;
	}

	
	/*
	public float[][] getTfIdfMatrix() throws Exception
 {
		IndexReader reader = DirectoryReader.open(mIndex);

		Map<String, Float> docFrequencies = new HashMap<String, Float>();

		
		
		Map<String, Integer> term_to_id = new HashMap<String, Integer>();

		MyTFIDFSimilarity tfidfSIM = new MyTFIDFSimilarity();
		Terms terms = SlowCompositeReaderWrapper.wrap(reader).terms("body"); 
		//terms.iterator(arg0)
		
		TermsEnum termsEnumIt = MultiFields.getTerms(reader, "body").iterator(null);
		 
		int number_of_terms =0 ;
		
		while (termsEnumIt.next() != null) {
			++number_of_terms;
		}
		//long temp2 = MultiFields.getTerms(reader, "body").getSumTotalTermFreq();
		//long temp = terms.size();
		//long number_of_terms = MultiFields.getTerms(reader, "body").size();
		long number_of_docs = reader.maxDoc();
		if((number_of_terms != (int)number_of_terms) || (number_of_docs  !=(int)number_of_docs)) 
		{
			throw new Exception("error while trying to create  tf-idf matrix");
		}
		
		float[][] tf_idf_matrix = new float[(int) number_of_terms][(int) number_of_docs];
		//
		
		
		
		
		
		TermsEnum termEnum = MultiFields.getTerms(reader, "body")
				.iterator(null);
		BytesRef bytesRef;
		int term_id_i = 0;
		while ((bytesRef = termEnum.next()) != null) {
			if (termEnum.seekExact(bytesRef)) {
				String term = bytesRef.utf8ToString();
				float idf = tfidfSIM.idf(termEnum.docFreq(),
						reader.numDocs());
				docFrequencies.put(term, idf);
				term_to_id.put(term, term_id_i);
				term_id_i++;
			}
		}

		for (int docID = 0; docID < reader.maxDoc(); docID++) {
			Map<String, Float> termFrequencies = new HashMap<String, Float>();
			//Map<String, Float> tf_Idf_Weights = new HashMap<String, Float>();
			TermsEnum termsEnum = MultiFields.getTerms(reader, "body")
					.iterator(null);
			DocsEnum docsEnum = null;

			Terms vector = reader.getTermVector(docID, "body");
			
			
			try {
				termsEnum = vector.iterator(termsEnum);
			} catch (NullPointerException e) {
				e.printStackTrace();
			}

			while ((bytesRef = termsEnum.next()) != null) {
				if (termsEnum.seekExact(bytesRef)) {
					String term = bytesRef.utf8ToString();
					float tf = 0;

					docsEnum = termsEnum.docs(null, null, DocsEnum.FLAG_FREQS);
					while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
						tf = tfidfSIM.tf(docsEnum.freq());
						termFrequencies.put(term, tf);
					}

					float idf = docFrequencies.get(term);
					float w = tf * idf;
					//tf_Idf_Weights.put(term, w);
					int term_id = term_to_id.get(term);
					tf_idf_matrix[term_id][docID] = w; 
				}
			}
		}
		return tf_idf_matrix;
	}
	

	*/
	/*
	public float get_distance_between_two_docs(int doc_id1, int doc_id2)
	{
		IndexReader reader = DirectoryReader.open(mIndex);

		Map<String, Float> docFrequencies = new HashMap<String, Float>();

		
		
		Map<String, Integer> term_to_id = new HashMap<String, Integer>();

		MyTFIDFSimilarity tfidfSIM = new MyTFIDFSimilarity();
		Terms terms = SlowCompositeReaderWrapper.wrap(reader).terms("body"); 
		//terms.iterator(arg0)
		
		TermsEnum termsEnumIt = MultiFields.getTerms(reader, "body").iterator(null);
		 
		int number_of_terms =0 ;
		
		while (termsEnumIt.next() != null) {
			++number_of_terms;
		}
		//long temp2 = MultiFields.getTerms(reader, "body").getSumTotalTermFreq();
		//long temp = terms.size();
		//long number_of_terms = MultiFields.getTerms(reader, "body").size();
		long number_of_docs = reader.maxDoc();
		if((number_of_terms != (int)number_of_terms) || (number_of_docs  !=(int)number_of_docs)) 
		{
			throw new Exception("error while trying to create  tf-idf matrix");
		}
		
		float[][] tf_idf_matrix = new float[(int) number_of_terms][(int) number_of_docs];
		//
		
		
		
		
		
		TermsEnum termEnum = MultiFields.getTerms(reader, "body")
				.iterator(null);
		BytesRef bytesRef;
		int term_id_i = 0;
		while ((bytesRef = termEnum.next()) != null) {
			if (termEnum.seekExact(bytesRef)) {
				String term = bytesRef.utf8ToString();
				float idf = tfidfSIM.idf(termEnum.docFreq(),
						reader.numDocs());
				docFrequencies.put(term, idf);
				term_to_id.put(term, term_id_i);
				term_id_i++;
			}
		}

		//for (int docID = 0; docID < reader.maxDoc(); docID++) 
		{
			Map<String, Float> termFrequencies = new HashMap<String, Float>();
			//Map<String, Float> tf_Idf_Weights = new HashMap<String, Float>();
			TermsEnum termsEnum = MultiFields.getTerms(reader, "body")
					.iterator(null);
			DocsEnum docsEnum = null;

			Terms vector = reader.getTermVector(docID, "body");

			
			try {
				termsEnum = vector.iterator(termsEnum);
			} catch (NullPointerException e) {
				e.printStackTrace();
			}

			while ((bytesRef = termsEnum.next()) != null) {
				if (termsEnum.seekExact(bytesRef)) {
					String term = bytesRef.utf8ToString();
					float tf = 0;

					docsEnum = termsEnum.docs(null, null, DocsEnum.FLAG_FREQS);
					while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
						tf = tfidfSIM.tf(docsEnum.freq());
						termFrequencies.put(term, tf);
					}

					float idf = docFrequencies.get(term);
					float w = tf * idf;
					//tf_Idf_Weights.put(term, w);
					int term_id = term_to_id.get(term);
					tf_idf_matrix[term_id][docID] = w; 
				}
			}
		}
		return tf_idf_matrix;
	}
	
	public HashMap<String, Float> get_tf_idf_vector(int doc_id) throws Exception
	{
		IndexReader reader = DirectoryReader.open(mIndex);

		Map<String, Float> docFrequencies = new HashMap<String, Float>();

		
		
		Map<String, Integer> term_to_id = new HashMap<String, Integer>();

		MyTFIDFSimilarity tfidfSIM = new MyTFIDFSimilarity();
		Terms terms = SlowCompositeReaderWrapper.wrap(reader).terms("body"); 
		//terms.iterator(arg0)
		
		TermsEnum termsEnumIt = MultiFields.getTerms(reader, "body").iterator(null);
		 
		int number_of_terms =0 ;
		
		while (termsEnumIt.next() != null) {
			++number_of_terms;
		}
		//long temp2 = MultiFields.getTerms(reader, "body").getSumTotalTermFreq();
		//long temp = terms.size();
		//long number_of_terms = MultiFields.getTerms(reader, "body").size();
		long number_of_docs = reader.maxDoc();
		if((number_of_terms != (int)number_of_terms) || (number_of_docs  !=(int)number_of_docs)) 
		{
			throw new Exception("error while trying to create  tf-idf matrix");
		}
		
		HashMap<String, Float> tf_idf_vector = new HashMap<String, Float>();
		//float[] tf_idf_matrix = new float[(int) number_of_terms][(int) number_of_docs];
		//
		
		
		
		
		
		TermsEnum termEnum = MultiFields.getTerms(reader, "body")
				.iterator(null);
		BytesRef bytesRef;
		int term_id_i = 0;
		
		while ((bytesRef = termEnum.next()) != null) {
			if (termEnum.seekExact(bytesRef)) {
				String term = bytesRef.utf8ToString();
				float idf = tfidfSIM.idf(termEnum.docFreq(),
						reader.numDocs());
				docFrequencies.put(term, idf);
				term_to_id.put(term, term_id_i);
				term_id_i++;
			}
		}

		//for (int docID = 0; docID < reader.maxDoc(); docID++) 
		{
			//Map<String, Float> termFrequencies = new HashMap<String, Float>();
			//Map<String, Float> tf_Idf_Weights = new HashMap<String, Float>();
			TermsEnum termsEnum = MultiFields.getTerms(reader, "body").iterator(null);
			DocsEnum docsEnum = null;
			
			Terms vector = reader.getTermVector(doc_id, "body");
			//termsEnum.docFreq()
			//.getTermVector(doc_id, "body").iterator(null).totalTermFreq()
			//reader.

			
			try {
				termsEnum = vector.iterator(termsEnum);
			} catch (NullPointerException e) {
				e.printStackTrace();
			}

			while ((bytesRef = termsEnum.next()) != null) {
				if (termsEnum.seekExact(bytesRef)) {
					String term = bytesRef.utf8ToString();
					float tf = 0;

					//what is the frequencey of the term in the current doc?
					docsEnum = termsEnum.docs(null, null, DocsEnum.FLAG_FREQS);
					while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
						tf = tfidfSIM.tf(docsEnum.freq());
						//termFrequencies.put(term, tf);
					}
					//termsEnum.docFreq()

					float idf = docFrequencies.get(term);
					float w = tf * idf;

					//tf_Idf_Weights.put(term, w);
					tf_idf_vector.put(term, w);
				}
			}
		}
		return tf_idf_vector;
	}
*/

}